{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0-dev20190317'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as K\n",
    "import numpy as np\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = K.layers.Input(shape=(None,None,3))\n",
    "conv1 = K.layers.Conv2D(32,3,strides=2, padding='same', use_bias=False)(inp)\n",
    "conv2 = K.layers.Conv2D(3,3,strides=2, padding='same', use_bias=False)(conv1)\n",
    "b_shape = tf.shape(conv2)\n",
    "tf.keras.layers.UpSampling2D()\n",
    "up = K.layers.Lambda(\n",
    "    lambda x: tf.image.resize(    #tf.image.resize(\n",
    "        x,b_shape[1:3]*tf.constant((2,2))))(conv2)\n",
    "test_model_new = K.Model(inputs=inp, outputs=up)\n",
    "\n",
    "inp = K.layers.Input(shape=(None,None,3))\n",
    "conv1 = K.layers.Conv2D(32,3,strides=2, padding='same', use_bias=False)(inp)\n",
    "conv2 = K.layers.Conv2D(3,3,strides=2, padding='same', use_bias=False)(conv1)\n",
    "b_shape = tf.shape(conv2)\n",
    "tf.keras.layers.UpSampling2D()\n",
    "up = K.layers.Lambda(\n",
    "    lambda x: tf.compat.v1.image.resize(\n",
    "        x, b_shape[1:3]*tf.constant((2,2)),method='bilinear'))(conv2)\n",
    "\n",
    "up = K.layers.GlobalAveragePooling2D()(conv2)\n",
    "tf.expand_dims()\n",
    "\n",
    "test_model_old = K.Model(inputs=inp, outputs=up)\n",
    "\n",
    "#to check that it works\n",
    "test_model_new.layers[1].weights[0].assign(test_model_old.layers[1].weights[0])\n",
    "test_model_new.layers[2].weights[0].assign(test_model_old.layers[2].weights[0])\n",
    "\n",
    "test_model_new_2 = K.Model(inputs=test_model_new.inputs,outputs=test_model_new.layers[2].output)\n",
    "test_model_old_2 = K.Model(inputs=test_model_old.inputs,outputs=test_model_old.layers[2].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dir(test_model_new.layers[1].weights[0].as)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model_new.compile(K.optimizers.SGD(),loss=K.losses.binary_crossentropy)\n",
    "test_model_old.compile(K.optimizers.SGD(),loss=K.losses.binary_crossentropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 32, 32, 3)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img = np.ones((1,64,64,3))\n",
    "out1 = test_model_new.predict(img)\n",
    "out2 = test_model_old.predict(img)\n",
    "out1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\"\"\" Deeplabv3+ model for Keras.\n",
    "This model is based on TF repo:\n",
    "https://github.com/tensorflow/models/tree/master/research/deeplab\n",
    "On Pascal VOC, original model gets to 84.56% mIOU\n",
    "\n",
    "Now this model is only available for the TensorFlow backend,\n",
    "due to its reliance on `SeparableConvolution` layers, but Theano will add\n",
    "this layer soon.\n",
    "\n",
    "MobileNetv2 backbone is based on this repo:\n",
    "https://github.com/JonathanCMitchell/mobilenet_v2_keras\n",
    "\n",
    "# Reference\n",
    "- [Encoder-Decoder with Atrous Separable Convolution\n",
    "    for Semantic Image Segmentation](https://arxiv.org/pdf/1802.02611.pdf)\n",
    "- [Xception: Deep Learning with Depthwise Separable Convolutions]\n",
    "    (https://arxiv.org/abs/1610.02357)\n",
    "- [Inverted Residuals and Linear Bottlenecks: Mobile Networks for\n",
    "    Classification, Detection and Segmentation](https://arxiv.org/abs/1801.04381)\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from tf.keras.models import Model\n",
    "from tf.keras import layers\n",
    "from tf.keras.layers import Input, Lambda\n",
    "from tf.keras.layers import Activation\n",
    "from tf.keras.layers import Concatenate\n",
    "from tf.keras.layers import Add\n",
    "from tf.keras.layers import Dropout\n",
    "from tf.keras.layers import BatchNormalization\n",
    "from tf.keras.layers import Conv2D\n",
    "from tf.keras.layers import DepthwiseConv2D\n",
    "from tf.keras.layers import ZeroPadding2D\n",
    "from tf.keras.layers import AveragePooling2D, GlobalAveragePooling2D\n",
    "from tf.keras.layers import Layer\n",
    "from tf.keras.layers import InputSpec\n",
    "from tf.python.keras.utils.layer_utils import get_source_inputs\n",
    "from tf.python.keras.applications.imagenet_utils import preprocess_input\n",
    "from tf.python.keras.utils import conv_utils\n",
    "from tf.python.keras.utils.data_utils import get_file\n",
    "WEIGHTS_PATH_X = \"https://github.com/bonlime/keras-deeplab-v3-plus/releases/download/1.1/deeplabv3_xception_tf_dim_ordering_tf_kernels.h5\"\n",
    "WEIGHTS_PATH_MOBILE = \"https://github.com/bonlime/keras-deeplab-v3-plus/releases/download/1.1/deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#class BilinearUpsampling(Layer):\n",
    "#    \"\"\"Just a simple bilinear upsampling layer. Works only with TF.\n",
    "#       Args:\n",
    "#           upsampling: tuple of 2 numbers > 0. The upsampling ratio for h and w\n",
    "#           output_size: used instead of upsampling arg if passed!\n",
    "#    \"\"\"\n",
    "#\n",
    "#    def __init__(self, upsampling=(2, 2), output_size=None, data_format=None, **kwargs):\n",
    "#\n",
    "#        super(BilinearUpsampling, self).__init__(**kwargs)\n",
    "#\n",
    "#        self.data_format = conv_utils.normalize_data_format(data_format)\n",
    "#        self.input_spec = InputSpec(ndim=4)\n",
    "#        if output_size:\n",
    "#            self.output_size = conv_utils.normalize_tuple(\n",
    "#                output_size, 2, 'output_size')\n",
    "#            self.upsampling = None\n",
    "#        else:\n",
    "#            self.output_size = None\n",
    "#            self.upsampling = conv_utils.normalize_tuple(\n",
    "#                upsampling, 2, 'upsampling')\n",
    "#\n",
    "#    def compute_output_shape(self, input_shape):\n",
    "#        if self.upsampling:\n",
    "#            height = self.upsampling[0] * \\\n",
    "#                input_shape[1] if input_shape[1] is not None else None\n",
    "#            width = self.upsampling[1] * \\\n",
    "#                input_shape[2] if input_shape[2] is not None else None\n",
    "#        else:\n",
    "#            height = self.output_size[0]\n",
    "#            width = self.output_size[1]\n",
    "#        return (input_shape[0],\n",
    "#                height,\n",
    "#                width,\n",
    "#                input_shape[3])\n",
    "#\n",
    "#    def call(self, inputs):\n",
    "#        if self.upsampling:\n",
    "#            return tf.compat.v1.image.resize_bilinear(inputs, (inputs.shape[1] * self.upsampling[0],\n",
    "#                                                       inputs.shape[2] * self.upsampling[1]),\n",
    "#                                              align_corners=True)\n",
    "#        else:\n",
    "#            return tf.compat.v1.image.resize_bilinear(inputs, (self.output_size[0],\n",
    "#                                                       self.output_size[1]),\n",
    "#                                              align_corners=True)\n",
    "#\n",
    "#    def get_config(self):\n",
    "#        config = {'upsampling': self.upsampling,\n",
    "#                  'output_size': self.output_size,\n",
    "#                  'data_format': self.data_format}\n",
    "#        base_config = super(BilinearUpsampling, self).get_config()\n",
    "#        return dict(list(base_config.items()) + list(config.items()))\n",
    "\n",
    "\n",
    "def SepConv_BN(x, filters, prefix, stride=1, kernel_size=3, rate=1, depth_activation=False, epsilon=1e-3):\n",
    "    \"\"\" SepConv with BN between depthwise & pointwise. Optionally add activation after BN\n",
    "        Implements right \"same\" padding for even kernel sizes\n",
    "        Args:\n",
    "            x: input tensor\n",
    "            filters: num of filters in pointwise convolution\n",
    "            prefix: prefix before name\n",
    "            stride: stride at depthwise conv\n",
    "            kernel_size: kernel size for depthwise convolution\n",
    "            rate: atrous rate for depthwise convolution\n",
    "            depth_activation: flag to use activation between depthwise & poinwise convs\n",
    "            epsilon: epsilon to use in BN layer\n",
    "    \"\"\"\n",
    "\n",
    "    if stride == 1:\n",
    "        depth_padding = 'same'\n",
    "    else:\n",
    "        kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)\n",
    "        pad_total = kernel_size_effective - 1\n",
    "        pad_beg = pad_total // 2\n",
    "        pad_end = pad_total - pad_beg\n",
    "        x = ZeroPadding2D((pad_beg, pad_end))(x)\n",
    "        depth_padding = 'valid'\n",
    "\n",
    "    if not depth_activation:\n",
    "        x = Activation('relu')(x)\n",
    "    x = DepthwiseConv2D((kernel_size, kernel_size), strides=(stride, stride), dilation_rate=(rate, rate),\n",
    "                        padding=depth_padding, use_bias=False, name=prefix + '_depthwise')(x)\n",
    "    x = BatchNormalization(name=prefix + '_depthwise_BN', epsilon=epsilon)(x)\n",
    "    if depth_activation:\n",
    "        x = Activation('relu')(x)\n",
    "    x = Conv2D(filters, (1, 1), padding='same',\n",
    "               use_bias=False, name=prefix + '_pointwise')(x)\n",
    "    x = BatchNormalization(name=prefix + '_pointwise_BN', epsilon=epsilon)(x)\n",
    "    if depth_activation:\n",
    "        x = Activation('relu')(x)\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def _conv2d_same(x, filters, prefix, stride=1, kernel_size=3, rate=1):\n",
    "    \"\"\"Implements right 'same' padding for even kernel sizes\n",
    "        Without this there is a 1 pixel drift when stride = 2\n",
    "        Args:\n",
    "            x: input tensor\n",
    "            filters: num of filters in pointwise convolution\n",
    "            prefix: prefix before name\n",
    "            stride: stride at depthwise conv\n",
    "            kernel_size: kernel size for depthwise convolution\n",
    "            rate: atrous rate for depthwise convolution\n",
    "    \"\"\"\n",
    "    if stride == 1:\n",
    "        return Conv2D(filters,\n",
    "                      (kernel_size, kernel_size),\n",
    "                      strides=(stride, stride),\n",
    "                      padding='same', use_bias=False,\n",
    "                      dilation_rate=(rate, rate),\n",
    "                      name=prefix)(x)\n",
    "    else:\n",
    "        kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)\n",
    "        pad_total = kernel_size_effective - 1\n",
    "        pad_beg = pad_total // 2\n",
    "        pad_end = pad_total - pad_beg\n",
    "        x = ZeroPadding2D((pad_beg, pad_end))(x)\n",
    "        return Conv2D(filters,\n",
    "                      (kernel_size, kernel_size),\n",
    "                      strides=(stride, stride),\n",
    "                      padding='valid', use_bias=False,\n",
    "                      dilation_rate=(rate, rate),\n",
    "                      name=prefix)(x)\n",
    "\n",
    "\n",
    "def _xception_block(inputs, depth_list, prefix, skip_connection_type, stride,\n",
    "                    rate=1, depth_activation=False, return_skip=False):\n",
    "    \"\"\" Basic building block of modified Xception network\n",
    "        Args:\n",
    "            inputs: input tensor\n",
    "            depth_list: number of filters in each SepConv layer. len(depth_list) == 3\n",
    "            prefix: prefix before name\n",
    "            skip_connection_type: one of {'conv','sum','none'}\n",
    "            stride: stride at last depthwise conv\n",
    "            rate: atrous rate for depthwise convolution\n",
    "            depth_activation: flag to use activation between depthwise & pointwise convs\n",
    "            return_skip: flag to return additional tensor after 2 SepConvs for decoder\n",
    "            \"\"\"\n",
    "    residual = inputs\n",
    "    for i in range(3):\n",
    "        residual = SepConv_BN(residual,\n",
    "                              depth_list[i],\n",
    "                              prefix + '_separable_conv{}'.format(i + 1),\n",
    "                              stride=stride if i == 2 else 1,\n",
    "                              rate=rate,\n",
    "                              depth_activation=depth_activation)\n",
    "        if i == 1:\n",
    "            skip = residual\n",
    "    if skip_connection_type == 'conv':\n",
    "        shortcut = _conv2d_same(inputs, depth_list[-1], prefix + '_shortcut',\n",
    "                                kernel_size=1,\n",
    "                                stride=stride)\n",
    "        shortcut = BatchNormalization(name=prefix + '_shortcut_BN')(shortcut)\n",
    "        outputs = layers.add([residual, shortcut])\n",
    "    elif skip_connection_type == 'sum':\n",
    "        outputs = layers.add([residual, inputs])\n",
    "    elif skip_connection_type == 'none':\n",
    "        outputs = residual\n",
    "    if return_skip:\n",
    "        return outputs, skip\n",
    "    else:\n",
    "        return outputs\n",
    "\n",
    "\n",
    "def relu6(x):\n",
    "    return keras.activations.relu(x, max_value=6)\n",
    "\n",
    "\n",
    "def _make_divisible(v, divisor, min_value=None):\n",
    "    if min_value is None:\n",
    "        min_value = divisor\n",
    "    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n",
    "    # Make sure that round down does not go down by more than 10%.\n",
    "    if new_v < 0.9 * v:\n",
    "        new_v += divisor\n",
    "    return new_v\n",
    "\n",
    "\n",
    "def _inverted_res_block(inputs, expansion, stride, alpha, filters, block_id, skip_connection, rate=1):\n",
    "    in_channels =  inputs.shape[-1] #inputs._keras_shape[-1]\n",
    "    pointwise_conv_filters = int(filters * alpha)\n",
    "    pointwise_filters = _make_divisible(pointwise_conv_filters, 8)\n",
    "    x = inputs\n",
    "    prefix = 'expanded_conv_{}_'.format(block_id)\n",
    "    if block_id:\n",
    "        # Expand\n",
    "\n",
    "        x = Conv2D(expansion * in_channels, kernel_size=1, padding='same',\n",
    "                   use_bias=False, activation=None,\n",
    "                   name=prefix + 'expand')(x)\n",
    "        x = BatchNormalization(epsilon=1e-3, momentum=0.999,\n",
    "                               name=prefix + 'expand_BN')(x)\n",
    "        x = Activation(relu6, name=prefix + 'expand_relu')(x)\n",
    "    else:\n",
    "        prefix = 'expanded_conv_'\n",
    "    # Depthwise\n",
    "    x = DepthwiseConv2D(kernel_size=3, strides=stride, activation=None,\n",
    "                        use_bias=False, padding='same', dilation_rate=(rate, rate),\n",
    "                        name=prefix + 'depthwise')(x)\n",
    "    x = BatchNormalization(epsilon=1e-3, momentum=0.999,\n",
    "                           name=prefix + 'depthwise_BN')(x)\n",
    "\n",
    "    x = Activation(relu6, name=prefix + 'depthwise_relu')(x)\n",
    "\n",
    "    # Project\n",
    "    x = Conv2D(pointwise_filters,\n",
    "               kernel_size=1, padding='same', use_bias=False, activation=None,\n",
    "               name=prefix + 'project')(x)\n",
    "    x = BatchNormalization(epsilon=1e-3, momentum=0.999,\n",
    "                           name=prefix + 'project_BN')(x)\n",
    "\n",
    "    if skip_connection:\n",
    "        return Add(name=prefix + 'add')([inputs, x])\n",
    "\n",
    "    # if in_channels == pointwise_filters and stride == 1:\n",
    "    #    return Add(name='res_connect_' + str(block_id))([inputs, x])\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def Deeplabv3(weights='pascal_voc', input_tensor=None, input_shape=(512, 512, 3), classes=21, backbone='mobilenetv2', OS=16, alpha=1.):\n",
    "    \"\"\" Instantiates the Deeplabv3+ architecture\n",
    "\n",
    "    Optionally loads weights pre-trained\n",
    "    on PASCAL VOC. This model is available for TensorFlow only,\n",
    "    and can only be used with inputs following the TensorFlow\n",
    "    data format `(width, height, channels)`.\n",
    "    # Arguments\n",
    "        weights: one of 'pascal_voc' (pre-trained on pascal voc)\n",
    "            or None (random initialization)\n",
    "        input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)\n",
    "            to use as image input for the model.\n",
    "        input_shape: shape of input image. format HxWxC\n",
    "            PASCAL VOC model was trained on (512,512,3) images\n",
    "        classes: number of desired classes. If classes != 21,\n",
    "            last layer is initialized randomly\n",
    "        backbone: backbone to use. one of {'xception','mobilenetv2'}\n",
    "        OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.\n",
    "            Used only for xception backbone.\n",
    "        alpha: controls the width of the MobileNetV2 network. This is known as the\n",
    "            width multiplier in the MobileNetV2 paper.\n",
    "                - If `alpha` < 1.0, proportionally decreases the number\n",
    "                    of filters in each layer.\n",
    "                - If `alpha` > 1.0, proportionally increases the number\n",
    "                    of filters in each layer.\n",
    "                - If `alpha` = 1, default number of filters from the paper\n",
    "                    are used at each layer.\n",
    "            Used only for mobilenetv2 backbone\n",
    "\n",
    "    # Returns\n",
    "        A Keras model instance.\n",
    "\n",
    "    # Raises\n",
    "        RuntimeError: If attempting to run this model with a\n",
    "            backend that does not support separable convolutions.\n",
    "        ValueError: in case of invalid argument for `weights` or `backbone`\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    if not (weights in {'pascal_voc', None}):\n",
    "        raise ValueError('The `weights` argument should be either '\n",
    "                         '`None` (random initialization) or `pascal_voc` '\n",
    "                         '(pre-trained on PASCAL VOC)')\n",
    "\n",
    "    #if K.backend() != 'tensorflow':\n",
    "    #    raise RuntimeError('The Deeplabv3+ model is only available with '\n",
    "    #                      'the TensorFlow backend.')\n",
    "\n",
    "    if not (backbone in {'xception', 'mobilenetv2'}):\n",
    "        raise ValueError('The `backbone` argument should be either '\n",
    "                         '`xception`  or `mobilenetv2` ')\n",
    "\n",
    "    if input_tensor is None:\n",
    "        img_input = Input(shape=input_shape)\n",
    "    else:\n",
    "        img_input = input_tensor\n",
    "        # not sure it's right though!\n",
    "        #if not K.is_keras_tensor(input_tensor):#\n",
    "        #    img_input = Input(tensor=input_tensor, shape=input_shape)\n",
    "        #else:\n",
    "        #    img_input = input_tensor\n",
    "    \n",
    "    if backbone == 'xception':\n",
    "        if OS == 8:\n",
    "            entry_block3_stride = 1\n",
    "            middle_block_rate = 2  # ! Not mentioned in paper, but required\n",
    "            exit_block_rates = (2, 4)\n",
    "            atrous_rates = (12, 24, 36)\n",
    "        else:\n",
    "            entry_block3_stride = 2\n",
    "            middle_block_rate = 1\n",
    "            exit_block_rates = (1, 2)\n",
    "            atrous_rates = (6, 12, 18)\n",
    "\n",
    "        x = Conv2D(32, (3, 3), strides=(2, 2),\n",
    "                   name='entry_flow_conv1_1', use_bias=False, padding='same')(img_input)\n",
    "        x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)\n",
    "        x = Activation('relu')(x)\n",
    "\n",
    "        x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)\n",
    "        x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)\n",
    "        x = Activation('relu')(x)\n",
    "\n",
    "        x = _xception_block(x, [128, 128, 128], 'entry_flow_block1',\n",
    "                            skip_connection_type='conv', stride=2,\n",
    "                            depth_activation=False)\n",
    "        x, skip1 = _xception_block(x, [256, 256, 256], 'entry_flow_block2',\n",
    "                                   skip_connection_type='conv', stride=2,\n",
    "                                   depth_activation=False, return_skip=True)\n",
    "\n",
    "        x = _xception_block(x, [728, 728, 728], 'entry_flow_block3',\n",
    "                            skip_connection_type='conv', stride=entry_block3_stride,\n",
    "                            depth_activation=False)\n",
    "        for i in range(16):\n",
    "            x = _xception_block(x, [728, 728, 728], 'middle_flow_unit_{}'.format(i + 1),\n",
    "                                skip_connection_type='sum', stride=1, rate=middle_block_rate,\n",
    "                                depth_activation=False)\n",
    "\n",
    "        x = _xception_block(x, [728, 1024, 1024], 'exit_flow_block1',\n",
    "                            skip_connection_type='conv', stride=1, rate=exit_block_rates[0],\n",
    "                            depth_activation=False)\n",
    "        x = _xception_block(x, [1536, 1536, 2048], 'exit_flow_block2',\n",
    "                            skip_connection_type='none', stride=1, rate=exit_block_rates[1],\n",
    "                            depth_activation=True)\n",
    "\n",
    "    else:\n",
    "        OS = 8\n",
    "        first_block_filters = _make_divisible(32 * alpha, 8)\n",
    "        x = Conv2D(first_block_filters,\n",
    "                   kernel_size=3,\n",
    "                   strides=(2, 2), padding='same',\n",
    "                   use_bias=False, name='Conv')(img_input)\n",
    "        x = BatchNormalization(\n",
    "            epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)\n",
    "        x = Activation(relu6, name='Conv_Relu6')(x)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=16, alpha=alpha, stride=1,\n",
    "                                expansion=1, block_id=0, skip_connection=False)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=24, alpha=alpha, stride=2,\n",
    "                                expansion=6, block_id=1, skip_connection=False)\n",
    "        x = _inverted_res_block(x, filters=24, alpha=alpha, stride=1,\n",
    "                                expansion=6, block_id=2, skip_connection=True)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=32, alpha=alpha, stride=2,\n",
    "                                expansion=6, block_id=3, skip_connection=False)\n",
    "        x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1,\n",
    "                                expansion=6, block_id=4, skip_connection=True)\n",
    "        x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1,\n",
    "                                expansion=6, block_id=5, skip_connection=True)\n",
    "\n",
    "        # stride in block 6 changed from 2 -> 1, so we need to use rate = 2\n",
    "        x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1,  # 1!\n",
    "                                expansion=6, block_id=6, skip_connection=False)\n",
    "        x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=7, skip_connection=True)\n",
    "        x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=8, skip_connection=True)\n",
    "        x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=9, skip_connection=True)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=10, skip_connection=False)\n",
    "        x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=11, skip_connection=True)\n",
    "        x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2,\n",
    "                                expansion=6, block_id=12, skip_connection=True)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=2,  # 1!\n",
    "                                expansion=6, block_id=13, skip_connection=False)\n",
    "        x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4,\n",
    "                                expansion=6, block_id=14, skip_connection=True)\n",
    "        x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4,\n",
    "                                expansion=6, block_id=15, skip_connection=True)\n",
    "\n",
    "        x = _inverted_res_block(x, filters=320, alpha=alpha, stride=1, rate=4,\n",
    "                                expansion=6, block_id=16, skip_connection=False)\n",
    "\n",
    "    # end of feature extractor\n",
    "\n",
    "    # branching for Atrous Spatial Pyramid Pooling\n",
    "\n",
    "    # Image Feature branch\n",
    "    #out_shape = int(np.ceil(input_shape[0] / OS))\n",
    "    #b4 = AveragePooling2D(pool_size=(int(np.ceil(input_shape[0] / OS)), int(np.ceil(input_shape[1] / OS))))(x)\n",
    "    shape_before = tf.shape(x)\n",
    "    b4 = GlobalAveragePooling2D()(x)\n",
    "    b4 = tf.expand_dims(tf.expand_dims(b4, 1),1) # from (b_size, channels)->(b_size, 1, 1, channels)\n",
    "    b4 = Conv2D(256, (1, 1), padding='same',\n",
    "                use_bias=False, name='image_pooling')(b4)\n",
    "    b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)\n",
    "    b4 = Activation('relu')(b4)\n",
    "    b4 = Lambda(lambda x: tf.compat.v1.image.resize(x, shape_before[1:3], \n",
    "                                                    method='bilinear',align_corners=True))(b4) #upsample\n",
    "    #b4 = BilinearUpsampling((int(np.ceil(input_shape[0] / OS)), int(np.ceil(input_shape[1] / OS))))(b4)\n",
    "\n",
    "    # simple 1x1\n",
    "    b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)\n",
    "    b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)\n",
    "    b0 = Activation('relu', name='aspp0_activation')(b0)\n",
    "\n",
    "    # there are only 2 branches in mobilenetV2. not sure why\n",
    "    if backbone == 'xception':\n",
    "        # rate = 6 (12)\n",
    "        b1 = SepConv_BN(x, 256, 'aspp1',\n",
    "                        rate=atrous_rates[0], depth_activation=True, epsilon=1e-5)\n",
    "        # rate = 12 (24)\n",
    "        b2 = SepConv_BN(x, 256, 'aspp2',\n",
    "                        rate=atrous_rates[1], depth_activation=True, epsilon=1e-5)\n",
    "        # rate = 18 (36)\n",
    "        b3 = SepConv_BN(x, 256, 'aspp3',\n",
    "                        rate=atrous_rates[2], depth_activation=True, epsilon=1e-5)\n",
    "\n",
    "        # concatenate ASPP branches & project\n",
    "        x = Concatenate()([b4, b0, b1, b2, b3])\n",
    "    else:\n",
    "        x = Concatenate()([b4, b0])\n",
    "\n",
    "    x = Conv2D(256, (1, 1), padding='same',\n",
    "               use_bias=False, name='concat_projection')(x)\n",
    "    x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "\n",
    "    # DeepLab v.3+ decoder\n",
    "\n",
    "    if backbone == 'xception':\n",
    "        # Feature projection\n",
    "        # x4 (x2) block\n",
    "        x = Lambda(\n",
    "            lambda x: tf.compat.v1.image.resize(x,\n",
    "                                                tf.shape(x)[1:3]*tf.constant((self.OS/4,self.OS/4)),\n",
    "                                                method='bilinear',align_corners=True))(x) \n",
    "        #x = BilinearUpsampling(output_size=(int(np.ceil(input_shape[0] / 4)),\n",
    "        #                                    int(np.ceil(input_shape[1] / 4))))(x)\n",
    "        dec_skip1 = Conv2D(48, (1, 1), padding='same',\n",
    "                           use_bias=False, name='feature_projection0')(skip1)\n",
    "        dec_skip1 = BatchNormalization(\n",
    "            name='feature_projection0_BN', epsilon=1e-5)(dec_skip1)\n",
    "        dec_skip1 = Activation('relu')(dec_skip1)\n",
    "        x = Concatenate()([x, dec_skip1])\n",
    "        x = SepConv_BN(x, 256, 'decoder_conv0',\n",
    "                       depth_activation=True, epsilon=1e-5)\n",
    "        x = SepConv_BN(x, 256, 'decoder_conv1',\n",
    "                       depth_activation=True, epsilon=1e-5)\n",
    "\n",
    "    # you can use it with arbitary number of classes\n",
    "    if classes == 21:\n",
    "        last_layer_name = 'logits_semantic'\n",
    "    else:\n",
    "        last_layer_name = 'custom_logits_semantic'\n",
    "\n",
    "    x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name)(x)\n",
    "    #upsample\n",
    "    x = Lambda(lambda xx: tf.compat.v1.image.resize(xx,\n",
    "                                                   tf.shape(img_input)[1:3],\n",
    "                                                   method='bilinear',align_corners=True))(x) \n",
    "    #x = BilinearUpsampling(output_size=(input_shape[0], input_shape[1]))(x)\n",
    "\n",
    "    # Ensure that the model takes into account\n",
    "    # any potential predecessors of `input_tensor`.\n",
    "    if input_tensor is not None:\n",
    "        inputs = get_source_inputs(input_tensor)\n",
    "    else:\n",
    "        inputs = img_input\n",
    "\n",
    "    model = Model(inputs, x, name='deeplabv3+')\n",
    "\n",
    "    # load weights\n",
    "\n",
    "    if weights == 'pascal_voc':\n",
    "        if backbone == 'xception':\n",
    "            weights_path = get_file('deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',\n",
    "                                    WEIGHTS_PATH_X,\n",
    "                                    cache_subdir='models')\n",
    "            model.load_weights(weights_path, by_name=True)\n",
    "        elif backbone == 'mobilenetv2':\n",
    "            if alpha == 1:\n",
    "                weights_path = get_file('deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5',\n",
    "                                        WEIGHTS_PATH_MOBILE,\n",
    "                                        cache_subdir='models')\n",
    "                model.load_weights(weights_path, by_name=True)\n",
    "            else:\n",
    "                print(\"Weights not loaded\")\n",
    "    return model\n",
    "\n",
    "\n",
    "def preprocess_input(x):\n",
    "    \"\"\"Preprocesses a numpy array encoding a batch of images.\n",
    "    # Arguments\n",
    "        x: a 4D numpy array consists of RGB values within [0, 255].\n",
    "    # Returns\n",
    "        Input array scaled to [-1.,1.]\n",
    "    \"\"\"\n",
    "    return imagenet_utils.preprocess_input(x, mode='tf')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights not loaded\n"
     ]
    }
   ],
   "source": [
    "deeplab_m = Deeplabv3(input_shape=(None,None,3),alpha=1.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.applications import MobileNetV2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = MobileNetV2(alpha=1.4, input_shape=(224,224,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Conv1 (Conv2D)                  (None, 112, 112, 48) 1296        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bn_Conv1 (BatchNormalization)   (None, 112, 112, 48) 192         Conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_relu (ReLU)               (None, 112, 112, 48) 0           bn_Conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise (Depthw (None, 112, 112, 48) 432         Conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_BN (Bat (None, 112, 112, 48) 192         expanded_conv_depthwise[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_relu (R (None, 112, 112, 48) 0           expanded_conv_depthwise_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project (Conv2D)  (None, 112, 112, 24) 1152        expanded_conv_depthwise_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project_BN (Batch (None, 112, 112, 24) 96          expanded_conv_project[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand (Conv2D)         (None, 112, 112, 144 3456        expanded_conv_project_BN[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_BN (BatchNormali (None, 112, 112, 144 576         block_1_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_relu (ReLU)      (None, 112, 112, 144 0           block_1_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise (DepthwiseCon (None, 56, 56, 144)  1296        block_1_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_BN (BatchNorm (None, 56, 56, 144)  576         block_1_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_relu (ReLU)   (None, 56, 56, 144)  0           block_1_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project (Conv2D)        (None, 56, 56, 32)   4608        block_1_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project_BN (BatchNormal (None, 56, 56, 32)   128         block_1_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand (Conv2D)         (None, 56, 56, 192)  6144        block_1_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_BN (BatchNormali (None, 56, 56, 192)  768         block_2_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_relu (ReLU)      (None, 56, 56, 192)  0           block_2_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise (DepthwiseCon (None, 56, 56, 192)  1728        block_2_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_BN (BatchNorm (None, 56, 56, 192)  768         block_2_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_relu (ReLU)   (None, 56, 56, 192)  0           block_2_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project (Conv2D)        (None, 56, 56, 32)   6144        block_2_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project_BN (BatchNormal (None, 56, 56, 32)   128         block_2_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_add (Add)               (None, 56, 56, 32)   0           block_1_project_BN[0][0]         \n",
      "                                                                 block_2_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand (Conv2D)         (None, 56, 56, 192)  6144        block_2_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_BN (BatchNormali (None, 56, 56, 192)  768         block_3_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_relu (ReLU)      (None, 56, 56, 192)  0           block_3_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise (DepthwiseCon (None, 28, 28, 192)  1728        block_3_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_BN (BatchNorm (None, 28, 28, 192)  768         block_3_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_relu (ReLU)   (None, 28, 28, 192)  0           block_3_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project (Conv2D)        (None, 28, 28, 48)   9216        block_3_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project_BN (BatchNormal (None, 28, 28, 48)   192         block_3_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand (Conv2D)         (None, 28, 28, 288)  13824       block_3_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_BN (BatchNormali (None, 28, 28, 288)  1152        block_4_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_relu (ReLU)      (None, 28, 28, 288)  0           block_4_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise (DepthwiseCon (None, 28, 28, 288)  2592        block_4_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_BN (BatchNorm (None, 28, 28, 288)  1152        block_4_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_relu (ReLU)   (None, 28, 28, 288)  0           block_4_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project (Conv2D)        (None, 28, 28, 48)   13824       block_4_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project_BN (BatchNormal (None, 28, 28, 48)   192         block_4_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_add (Add)               (None, 28, 28, 48)   0           block_3_project_BN[0][0]         \n",
      "                                                                 block_4_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand (Conv2D)         (None, 28, 28, 288)  13824       block_4_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_BN (BatchNormali (None, 28, 28, 288)  1152        block_5_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_relu (ReLU)      (None, 28, 28, 288)  0           block_5_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise (DepthwiseCon (None, 28, 28, 288)  2592        block_5_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_BN (BatchNorm (None, 28, 28, 288)  1152        block_5_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_relu (ReLU)   (None, 28, 28, 288)  0           block_5_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project (Conv2D)        (None, 28, 28, 48)   13824       block_5_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project_BN (BatchNormal (None, 28, 28, 48)   192         block_5_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_5_add (Add)               (None, 28, 28, 48)   0           block_4_add[0][0]                \n",
      "                                                                 block_5_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand (Conv2D)         (None, 28, 28, 288)  13824       block_5_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_BN (BatchNormali (None, 28, 28, 288)  1152        block_6_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_relu (ReLU)      (None, 28, 28, 288)  0           block_6_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise (DepthwiseCon (None, 14, 14, 288)  2592        block_6_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_BN (BatchNorm (None, 14, 14, 288)  1152        block_6_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_relu (ReLU)   (None, 14, 14, 288)  0           block_6_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project (Conv2D)        (None, 14, 14, 88)   25344       block_6_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project_BN (BatchNormal (None, 14, 14, 88)   352         block_6_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand (Conv2D)         (None, 14, 14, 528)  46464       block_6_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_BN (BatchNormali (None, 14, 14, 528)  2112        block_7_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_relu (ReLU)      (None, 14, 14, 528)  0           block_7_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise (DepthwiseCon (None, 14, 14, 528)  4752        block_7_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_BN (BatchNorm (None, 14, 14, 528)  2112        block_7_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_relu (ReLU)   (None, 14, 14, 528)  0           block_7_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project (Conv2D)        (None, 14, 14, 88)   46464       block_7_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project_BN (BatchNormal (None, 14, 14, 88)   352         block_7_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_add (Add)               (None, 14, 14, 88)   0           block_6_project_BN[0][0]         \n",
      "                                                                 block_7_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand (Conv2D)         (None, 14, 14, 528)  46464       block_7_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_BN (BatchNormali (None, 14, 14, 528)  2112        block_8_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_relu (ReLU)      (None, 14, 14, 528)  0           block_8_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise (DepthwiseCon (None, 14, 14, 528)  4752        block_8_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_BN (BatchNorm (None, 14, 14, 528)  2112        block_8_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_relu (ReLU)   (None, 14, 14, 528)  0           block_8_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project (Conv2D)        (None, 14, 14, 88)   46464       block_8_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project_BN (BatchNormal (None, 14, 14, 88)   352         block_8_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_8_add (Add)               (None, 14, 14, 88)   0           block_7_add[0][0]                \n",
      "                                                                 block_8_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand (Conv2D)         (None, 14, 14, 528)  46464       block_8_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_BN (BatchNormali (None, 14, 14, 528)  2112        block_9_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_relu (ReLU)      (None, 14, 14, 528)  0           block_9_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise (DepthwiseCon (None, 14, 14, 528)  4752        block_9_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_BN (BatchNorm (None, 14, 14, 528)  2112        block_9_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_relu (ReLU)   (None, 14, 14, 528)  0           block_9_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project (Conv2D)        (None, 14, 14, 88)   46464       block_9_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project_BN (BatchNormal (None, 14, 14, 88)   352         block_9_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_9_add (Add)               (None, 14, 14, 88)   0           block_8_add[0][0]                \n",
      "                                                                 block_9_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand (Conv2D)        (None, 14, 14, 528)  46464       block_9_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_BN (BatchNormal (None, 14, 14, 528)  2112        block_10_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_relu (ReLU)     (None, 14, 14, 528)  0           block_10_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise (DepthwiseCo (None, 14, 14, 528)  4752        block_10_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_BN (BatchNor (None, 14, 14, 528)  2112        block_10_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_relu (ReLU)  (None, 14, 14, 528)  0           block_10_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project (Conv2D)       (None, 14, 14, 136)  71808       block_10_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project_BN (BatchNorma (None, 14, 14, 136)  544         block_10_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand (Conv2D)        (None, 14, 14, 816)  110976      block_10_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_BN (BatchNormal (None, 14, 14, 816)  3264        block_11_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_relu (ReLU)     (None, 14, 14, 816)  0           block_11_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise (DepthwiseCo (None, 14, 14, 816)  7344        block_11_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_BN (BatchNor (None, 14, 14, 816)  3264        block_11_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_relu (ReLU)  (None, 14, 14, 816)  0           block_11_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project (Conv2D)       (None, 14, 14, 136)  110976      block_11_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project_BN (BatchNorma (None, 14, 14, 136)  544         block_11_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_add (Add)              (None, 14, 14, 136)  0           block_10_project_BN[0][0]        \n",
      "                                                                 block_11_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand (Conv2D)        (None, 14, 14, 816)  110976      block_11_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_BN (BatchNormal (None, 14, 14, 816)  3264        block_12_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_relu (ReLU)     (None, 14, 14, 816)  0           block_12_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise (DepthwiseCo (None, 14, 14, 816)  7344        block_12_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_BN (BatchNor (None, 14, 14, 816)  3264        block_12_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_relu (ReLU)  (None, 14, 14, 816)  0           block_12_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project (Conv2D)       (None, 14, 14, 136)  110976      block_12_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project_BN (BatchNorma (None, 14, 14, 136)  544         block_12_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_12_add (Add)              (None, 14, 14, 136)  0           block_11_add[0][0]               \n",
      "                                                                 block_12_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand (Conv2D)        (None, 14, 14, 816)  110976      block_12_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_BN (BatchNormal (None, 14, 14, 816)  3264        block_13_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_relu (ReLU)     (None, 14, 14, 816)  0           block_13_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise (DepthwiseCo (None, 7, 7, 816)    7344        block_13_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_BN (BatchNor (None, 7, 7, 816)    3264        block_13_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_relu (ReLU)  (None, 7, 7, 816)    0           block_13_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project (Conv2D)       (None, 7, 7, 224)    182784      block_13_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project_BN (BatchNorma (None, 7, 7, 224)    896         block_13_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand (Conv2D)        (None, 7, 7, 1344)   301056      block_13_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_BN (BatchNormal (None, 7, 7, 1344)   5376        block_14_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_relu (ReLU)     (None, 7, 7, 1344)   0           block_14_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise (DepthwiseCo (None, 7, 7, 1344)   12096       block_14_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_BN (BatchNor (None, 7, 7, 1344)   5376        block_14_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_relu (ReLU)  (None, 7, 7, 1344)   0           block_14_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project (Conv2D)       (None, 7, 7, 224)    301056      block_14_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project_BN (BatchNorma (None, 7, 7, 224)    896         block_14_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_add (Add)              (None, 7, 7, 224)    0           block_13_project_BN[0][0]        \n",
      "                                                                 block_14_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand (Conv2D)        (None, 7, 7, 1344)   301056      block_14_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_BN (BatchNormal (None, 7, 7, 1344)   5376        block_15_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_relu (ReLU)     (None, 7, 7, 1344)   0           block_15_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise (DepthwiseCo (None, 7, 7, 1344)   12096       block_15_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_BN (BatchNor (None, 7, 7, 1344)   5376        block_15_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_relu (ReLU)  (None, 7, 7, 1344)   0           block_15_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project (Conv2D)       (None, 7, 7, 224)    301056      block_15_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project_BN (BatchNorma (None, 7, 7, 224)    896         block_15_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_15_add (Add)              (None, 7, 7, 224)    0           block_14_add[0][0]               \n",
      "                                                                 block_15_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand (Conv2D)        (None, 7, 7, 1344)   301056      block_15_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_BN (BatchNormal (None, 7, 7, 1344)   5376        block_16_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_relu (ReLU)     (None, 7, 7, 1344)   0           block_16_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise (DepthwiseCo (None, 7, 7, 1344)   12096       block_16_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_BN (BatchNor (None, 7, 7, 1344)   5376        block_16_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_relu (ReLU)  (None, 7, 7, 1344)   0           block_16_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project (Conv2D)       (None, 7, 7, 448)    602112      block_16_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project_BN (BatchNorma (None, 7, 7, 448)    1792        block_16_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1 (Conv2D)                 (None, 7, 7, 1792)   802816      block_16_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1_bn (BatchNormalization)  (None, 7, 7, 1792)   7168        Conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "out_relu (ReLU)                 (None, 7, 7, 1792)   0           Conv_1_bn[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 1792)         0           out_relu[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "Logits (Dense)                  (None, 1000)         1793000     global_average_pooling2d_2[0][0] \n",
      "==================================================================================================\n",
      "Total params: 6,156,712\n",
      "Trainable params: 6,108,776\n",
      "Non-trainable params: 47,936\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Conv (Conv2D)                   (None, 112, 112, 48) 1296        input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Conv_BN (BatchNormalization)    (None, 112, 112, 48) 192         Conv[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "Conv_Relu6 (Activation)         (None, 112, 112, 48) 0           Conv_BN[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise (Depthw (None, 112, 112, 48) 432         Conv_Relu6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_BN (Bat (None, 112, 112, 48) 192         expanded_conv_depthwise[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_relu (A (None, 112, 112, 48) 0           expanded_conv_depthwise_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project (Conv2D)  (None, 112, 112, 24) 1152        expanded_conv_depthwise_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project_BN (Batch (None, 112, 112, 24) 96          expanded_conv_project[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_expand (Conv2D) (None, 112, 112, 144 3456        expanded_conv_project_BN[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_expand_BN (Batc (None, 112, 112, 144 576         expanded_conv_1_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_expand_relu (Ac (None, 112, 112, 144 0           expanded_conv_1_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_depthwise (Dept (None, 56, 56, 144)  1296        expanded_conv_1_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_depthwise_BN (B (None, 56, 56, 144)  576         expanded_conv_1_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_depthwise_relu  (None, 56, 56, 144)  0           expanded_conv_1_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_project (Conv2D (None, 56, 56, 32)   4608        expanded_conv_1_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_1_project_BN (Bat (None, 56, 56, 32)   128         expanded_conv_1_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_expand (Conv2D) (None, 56, 56, 192)  6144        expanded_conv_1_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_expand_BN (Batc (None, 56, 56, 192)  768         expanded_conv_2_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_expand_relu (Ac (None, 56, 56, 192)  0           expanded_conv_2_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_depthwise (Dept (None, 56, 56, 192)  1728        expanded_conv_2_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_depthwise_BN (B (None, 56, 56, 192)  768         expanded_conv_2_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_depthwise_relu  (None, 56, 56, 192)  0           expanded_conv_2_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_project (Conv2D (None, 56, 56, 32)   6144        expanded_conv_2_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_project_BN (Bat (None, 56, 56, 32)   128         expanded_conv_2_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_2_add (Add)       (None, 56, 56, 32)   0           expanded_conv_1_project_BN[0][0] \n",
      "                                                                 expanded_conv_2_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_expand (Conv2D) (None, 56, 56, 192)  6144        expanded_conv_2_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_expand_BN (Batc (None, 56, 56, 192)  768         expanded_conv_3_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_expand_relu (Ac (None, 56, 56, 192)  0           expanded_conv_3_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_depthwise (Dept (None, 28, 28, 192)  1728        expanded_conv_3_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_depthwise_BN (B (None, 28, 28, 192)  768         expanded_conv_3_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_depthwise_relu  (None, 28, 28, 192)  0           expanded_conv_3_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_project (Conv2D (None, 28, 28, 48)   9216        expanded_conv_3_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_3_project_BN (Bat (None, 28, 28, 48)   192         expanded_conv_3_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_expand (Conv2D) (None, 28, 28, 288)  13824       expanded_conv_3_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_expand_BN (Batc (None, 28, 28, 288)  1152        expanded_conv_4_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_expand_relu (Ac (None, 28, 28, 288)  0           expanded_conv_4_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_depthwise (Dept (None, 28, 28, 288)  2592        expanded_conv_4_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_depthwise_BN (B (None, 28, 28, 288)  1152        expanded_conv_4_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_depthwise_relu  (None, 28, 28, 288)  0           expanded_conv_4_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_project (Conv2D (None, 28, 28, 48)   13824       expanded_conv_4_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_project_BN (Bat (None, 28, 28, 48)   192         expanded_conv_4_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_4_add (Add)       (None, 28, 28, 48)   0           expanded_conv_3_project_BN[0][0] \n",
      "                                                                 expanded_conv_4_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_expand (Conv2D) (None, 28, 28, 288)  13824       expanded_conv_4_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_expand_BN (Batc (None, 28, 28, 288)  1152        expanded_conv_5_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_expand_relu (Ac (None, 28, 28, 288)  0           expanded_conv_5_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_depthwise (Dept (None, 28, 28, 288)  2592        expanded_conv_5_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_depthwise_BN (B (None, 28, 28, 288)  1152        expanded_conv_5_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_depthwise_relu  (None, 28, 28, 288)  0           expanded_conv_5_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_project (Conv2D (None, 28, 28, 48)   13824       expanded_conv_5_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_project_BN (Bat (None, 28, 28, 48)   192         expanded_conv_5_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_5_add (Add)       (None, 28, 28, 48)   0           expanded_conv_4_add[0][0]        \n",
      "                                                                 expanded_conv_5_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_expand (Conv2D) (None, 28, 28, 288)  13824       expanded_conv_5_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_expand_BN (Batc (None, 28, 28, 288)  1152        expanded_conv_6_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_expand_relu (Ac (None, 28, 28, 288)  0           expanded_conv_6_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_depthwise (Dept (None, 28, 28, 288)  2592        expanded_conv_6_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_depthwise_BN (B (None, 28, 28, 288)  1152        expanded_conv_6_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_depthwise_relu  (None, 28, 28, 288)  0           expanded_conv_6_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_project (Conv2D (None, 28, 28, 88)   25344       expanded_conv_6_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_6_project_BN (Bat (None, 28, 28, 88)   352         expanded_conv_6_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_expand (Conv2D) (None, 28, 28, 528)  46464       expanded_conv_6_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_expand_BN (Batc (None, 28, 28, 528)  2112        expanded_conv_7_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_expand_relu (Ac (None, 28, 28, 528)  0           expanded_conv_7_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_depthwise (Dept (None, 28, 28, 528)  4752        expanded_conv_7_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_depthwise_BN (B (None, 28, 28, 528)  2112        expanded_conv_7_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_depthwise_relu  (None, 28, 28, 528)  0           expanded_conv_7_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_project (Conv2D (None, 28, 28, 88)   46464       expanded_conv_7_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_project_BN (Bat (None, 28, 28, 88)   352         expanded_conv_7_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_7_add (Add)       (None, 28, 28, 88)   0           expanded_conv_6_project_BN[0][0] \n",
      "                                                                 expanded_conv_7_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_expand (Conv2D) (None, 28, 28, 528)  46464       expanded_conv_7_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_expand_BN (Batc (None, 28, 28, 528)  2112        expanded_conv_8_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_expand_relu (Ac (None, 28, 28, 528)  0           expanded_conv_8_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_depthwise (Dept (None, 28, 28, 528)  4752        expanded_conv_8_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_depthwise_BN (B (None, 28, 28, 528)  2112        expanded_conv_8_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_depthwise_relu  (None, 28, 28, 528)  0           expanded_conv_8_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_project (Conv2D (None, 28, 28, 88)   46464       expanded_conv_8_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_project_BN (Bat (None, 28, 28, 88)   352         expanded_conv_8_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_8_add (Add)       (None, 28, 28, 88)   0           expanded_conv_7_add[0][0]        \n",
      "                                                                 expanded_conv_8_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_expand (Conv2D) (None, 28, 28, 528)  46464       expanded_conv_8_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_expand_BN (Batc (None, 28, 28, 528)  2112        expanded_conv_9_expand[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_expand_relu (Ac (None, 28, 28, 528)  0           expanded_conv_9_expand_BN[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_depthwise (Dept (None, 28, 28, 528)  4752        expanded_conv_9_expand_relu[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_depthwise_BN (B (None, 28, 28, 528)  2112        expanded_conv_9_depthwise[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_depthwise_relu  (None, 28, 28, 528)  0           expanded_conv_9_depthwise_BN[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_project (Conv2D (None, 28, 28, 88)   46464       expanded_conv_9_depthwise_relu[0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_project_BN (Bat (None, 28, 28, 88)   352         expanded_conv_9_project[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_9_add (Add)       (None, 28, 28, 88)   0           expanded_conv_8_add[0][0]        \n",
      "                                                                 expanded_conv_9_project_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_expand (Conv2D (None, 28, 28, 528)  46464       expanded_conv_9_add[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_expand_BN (Bat (None, 28, 28, 528)  2112        expanded_conv_10_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_expand_relu (A (None, 28, 28, 528)  0           expanded_conv_10_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_depthwise (Dep (None, 28, 28, 528)  4752        expanded_conv_10_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_depthwise_BN ( (None, 28, 28, 528)  2112        expanded_conv_10_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_depthwise_relu (None, 28, 28, 528)  0           expanded_conv_10_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_project (Conv2 (None, 28, 28, 136)  71808       expanded_conv_10_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_10_project_BN (Ba (None, 28, 28, 136)  544         expanded_conv_10_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_expand (Conv2D (None, 28, 28, 816)  110976      expanded_conv_10_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_expand_BN (Bat (None, 28, 28, 816)  3264        expanded_conv_11_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_expand_relu (A (None, 28, 28, 816)  0           expanded_conv_11_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_depthwise (Dep (None, 28, 28, 816)  7344        expanded_conv_11_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_depthwise_BN ( (None, 28, 28, 816)  3264        expanded_conv_11_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_depthwise_relu (None, 28, 28, 816)  0           expanded_conv_11_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_project (Conv2 (None, 28, 28, 136)  110976      expanded_conv_11_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_project_BN (Ba (None, 28, 28, 136)  544         expanded_conv_11_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_11_add (Add)      (None, 28, 28, 136)  0           expanded_conv_10_project_BN[0][0]\n",
      "                                                                 expanded_conv_11_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_expand (Conv2D (None, 28, 28, 816)  110976      expanded_conv_11_add[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_expand_BN (Bat (None, 28, 28, 816)  3264        expanded_conv_12_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_expand_relu (A (None, 28, 28, 816)  0           expanded_conv_12_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_depthwise (Dep (None, 28, 28, 816)  7344        expanded_conv_12_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_depthwise_BN ( (None, 28, 28, 816)  3264        expanded_conv_12_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_depthwise_relu (None, 28, 28, 816)  0           expanded_conv_12_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_project (Conv2 (None, 28, 28, 136)  110976      expanded_conv_12_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_project_BN (Ba (None, 28, 28, 136)  544         expanded_conv_12_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_12_add (Add)      (None, 28, 28, 136)  0           expanded_conv_11_add[0][0]       \n",
      "                                                                 expanded_conv_12_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_expand (Conv2D (None, 28, 28, 816)  110976      expanded_conv_12_add[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_expand_BN (Bat (None, 28, 28, 816)  3264        expanded_conv_13_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_expand_relu (A (None, 28, 28, 816)  0           expanded_conv_13_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_depthwise (Dep (None, 28, 28, 816)  7344        expanded_conv_13_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_depthwise_BN ( (None, 28, 28, 816)  3264        expanded_conv_13_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_depthwise_relu (None, 28, 28, 816)  0           expanded_conv_13_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_project (Conv2 (None, 28, 28, 224)  182784      expanded_conv_13_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_13_project_BN (Ba (None, 28, 28, 224)  896         expanded_conv_13_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_expand (Conv2D (None, 28, 28, 1344) 301056      expanded_conv_13_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_expand_BN (Bat (None, 28, 28, 1344) 5376        expanded_conv_14_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_expand_relu (A (None, 28, 28, 1344) 0           expanded_conv_14_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_depthwise (Dep (None, 28, 28, 1344) 12096       expanded_conv_14_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_depthwise_BN ( (None, 28, 28, 1344) 5376        expanded_conv_14_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_depthwise_relu (None, 28, 28, 1344) 0           expanded_conv_14_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_project (Conv2 (None, 28, 28, 224)  301056      expanded_conv_14_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_project_BN (Ba (None, 28, 28, 224)  896         expanded_conv_14_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_14_add (Add)      (None, 28, 28, 224)  0           expanded_conv_13_project_BN[0][0]\n",
      "                                                                 expanded_conv_14_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_expand (Conv2D (None, 28, 28, 1344) 301056      expanded_conv_14_add[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_expand_BN (Bat (None, 28, 28, 1344) 5376        expanded_conv_15_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_expand_relu (A (None, 28, 28, 1344) 0           expanded_conv_15_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_depthwise (Dep (None, 28, 28, 1344) 12096       expanded_conv_15_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_depthwise_BN ( (None, 28, 28, 1344) 5376        expanded_conv_15_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_depthwise_relu (None, 28, 28, 1344) 0           expanded_conv_15_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_project (Conv2 (None, 28, 28, 224)  301056      expanded_conv_15_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_project_BN (Ba (None, 28, 28, 224)  896         expanded_conv_15_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_15_add (Add)      (None, 28, 28, 224)  0           expanded_conv_14_add[0][0]       \n",
      "                                                                 expanded_conv_15_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_expand (Conv2D (None, 28, 28, 1344) 301056      expanded_conv_15_add[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_expand_BN (Bat (None, 28, 28, 1344) 5376        expanded_conv_16_expand[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_expand_relu (A (None, 28, 28, 1344) 0           expanded_conv_16_expand_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_depthwise (Dep (None, 28, 28, 1344) 12096       expanded_conv_16_expand_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_depthwise_BN ( (None, 28, 28, 1344) 5376        expanded_conv_16_depthwise[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_depthwise_relu (None, 28, 28, 1344) 0           expanded_conv_16_depthwise_BN[0][\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_project (Conv2 (None, 28, 28, 448)  602112      expanded_conv_16_depthwise_relu[0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_16_project_BN (Ba (None, 28, 28, 448)  1792        expanded_conv_16_project[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_5 (AveragePoo (None, 1, 1, 448)    0           expanded_conv_16_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "image_pooling (Conv2D)          (None, 1, 1, 256)    114688      average_pooling2d_5[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "image_pooling_BN (BatchNormaliz (None, 1, 1, 256)    1024        image_pooling[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "aspp0 (Conv2D)                  (None, 28, 28, 256)  114688      expanded_conv_16_project_BN[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 1, 1, 256)    0           image_pooling_BN[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "aspp0_BN (BatchNormalization)   (None, 28, 28, 256)  1024        aspp0[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "bilinear_upsampling_8 (Bilinear (None, 28, 28, 256)  0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "aspp0_activation (Activation)   (None, 28, 28, 256)  0           aspp0_BN[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 28, 28, 512)  0           bilinear_upsampling_8[0][0]      \n",
      "                                                                 aspp0_activation[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "concat_projection (Conv2D)      (None, 28, 28, 256)  131072      concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concat_projection_BN (BatchNorm (None, 28, 28, 256)  1024        concat_projection[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 28, 28, 256)  0           concat_projection_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 28, 28, 256)  0           activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "logits_semantic (Conv2D)        (None, 28, 28, 21)   5397        dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bilinear_upsampling_9 (Bilinear (None, 224, 224, 21) 0           logits_semantic[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 3,922,645\n",
      "Trainable params: 3,876,757\n",
      "Non-trainable params: 45,888\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "deeplab_m.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
